{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tarfile\n",
    "from six.moves import urllib\n",
    "\n",
    "DOWNLOAD_ROOT = \"http://spamassassin.apache.org/old/publiccorpus/\"\n",
    "HAM_URL = DOWNLOAD_ROOT + \"20030228_easy_ham.tar.bz2\"\n",
    "SPAM_URL = DOWNLOAD_ROOT + \"20030228_spam.tar.bz2\"\n",
    "SPAM_PATH = os.path.join(\"datasets\",\"spam\")\n",
    "\n",
    "def fetch_spam_data(spam_url = SPAM_URL,spam_path=SPAM_PATH):\n",
    "    if not os.path.isdir(spam_path):\n",
    "        os.makedirs(spam_path)\n",
    "    for filename,url in ((\"ham.tar.bz2\",HAM_URL),(\"spam.tar.bz2\",SPAM_URL)):\n",
    "        path = os.path.join(spam_path,filename)\n",
    "        if not os.path.isfile(path):\n",
    "            urllib.request.urlretrieve(url,path)\n",
    "        tar_bz2_file = tarfile.open(path)\n",
    "        tar_bz2_file.extractall(path=SPAM_PATH)\n",
    "        tar_bz2_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "fetch_spam_data()\n",
    "HAM_DIR = os.path.join(SPAM_PATH,\"easy_ham\")\n",
    "SPAM_DIR = os.path.join(SPAM_PATH,\"spam\")\n",
    "ham_filenames = [name for name in sorted(os.listdir(HAM_DIR)) if len(name)]\n",
    "spam_filenames = [name for name in sorted(os.listdir(SPAM_DIR)) if len(name)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 用pathon的‘email'模块解析这些邮件\n",
    "import email\n",
    "import email.policy\n",
    "\n",
    "def load_email(is_spam,filename,spam_path = SPAM_PATH):\n",
    "    directory = \"spam\" if is_spam else \"easy_ham\"\n",
    "    with open(os.path.join(spam_path,directory,filename),\"rb\") as f:\n",
    "        return email.parser.BytesParser(policy=email.policy.default).parse(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Martin A posted:\n",
      "Tassos Papadopoulos, the Greek sculptor behind the plan, judged that the\n",
      " limestone of Mount Kerdylio, 70 miles east of Salonika and not far from the\n",
      " Mount Athos monastic community, was ideal for the patriotic sculpture. \n",
      " \n",
      " As well as Alexander's granite features, 240 ft high and 170 ft wide, a\n",
      " museum, a restored amphitheatre and car park for admiring crowds are\n",
      "planned\n",
      "---------------------\n",
      "So is this mountain limestone or granite?\n",
      "If it's limestone, it'll weather pretty fast.\n",
      "\n",
      "------------------------ Yahoo! Groups Sponsor ---------------------~-->\n",
      "4 DVDs Free +s&p Join Now\n",
      "http://us.click.yahoo.com/pt6YBB/NXiEAA/mG3HAA/7gSolB/TM\n",
      "---------------------------------------------------------------------~->\n",
      "\n",
      "To unsubscribe from this group, send an email to:\n",
      "forteana-unsubscribe@egroups.com\n",
      "\n",
      " \n",
      "\n",
      "Your use of Yahoo! Groups is subject to http://docs.yahoo.com/info/terms/\n"
     ]
    }
   ],
   "source": [
    "# 让我们看一个ham示例和一个spam示例，了解数据的外观：\n",
    "ham_emails = [load_email(is_spam=False, filename=name) for name in ham_filenames]\n",
    "spam_emails = [load_email(is_spam=True, filename=name) for name in spam_filenames]\n",
    "print(ham_emails[1].get_content().strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help wanted.  We are a 14 year old fortune 500 company, that is\n",
      "growing at a tremendous rate.  We are looking for individuals who\n",
      "want to work from home.\n",
      "\n",
      "This is an opportunity to make an excellent income.  No experience\n",
      "is required.  We will train you.\n",
      "\n",
      "So if you are looking to be employed from home with a career that has\n",
      "vast opportunities, then go:\n",
      "\n",
      "http://www.basetel.com/wealthnow\n",
      "\n",
      "We are looking for energetic and self motivated people.  If that is you\n",
      "than click on the link and fill out the form, and one of our\n",
      "employement specialist will contact you.\n",
      "\n",
      "To be removed from our link simple go to:\n",
      "\n",
      "http://www.basetel.com/remove.html\n",
      "\n",
      "\n",
      "4139vOLW7-758DoDY1425FRhM1-764SMFc8513fCsLl40\n"
     ]
    }
   ],
   "source": [
    "print(spam_emails[6].get_content().strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_email_structure(email):\n",
    "    if isinstance(email,str):\n",
    "        return email\n",
    "    payload = email.get_payload()\n",
    "    if isinstance(payload,list):\n",
    "        return \"multipart({})\".format(\", \".join([\n",
    "            get_email_structure(sub_email)\n",
    "            for sub_email in payload\n",
    "        ]))\n",
    "    else:\n",
    "        return email.get_content_type()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "def structures_counter(emails):\n",
    "    structures = Counter()\n",
    "    for email in emails:\n",
    "        structure = get_email_structure(email)\n",
    "        structures[structure] += 1\n",
    "    return structures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('text/plain', 2409),\n",
       " ('multipart(text/plain, application/pgp-signature)', 66),\n",
       " ('multipart(text/plain, text/html)', 8),\n",
       " ('multipart(text/plain, text/plain)', 4),\n",
       " ('multipart(text/plain)', 3),\n",
       " ('multipart(text/plain, application/octet-stream)', 2),\n",
       " ('multipart(text/plain, text/enriched)', 1),\n",
       " ('multipart(text/plain, application/ms-tnef, text/plain)', 1),\n",
       " ('multipart(multipart(text/plain, text/plain, text/plain), application/pgp-signature)',\n",
       "  1),\n",
       " ('multipart(text/plain, video/mng)', 1),\n",
       " ('multipart(text/plain, multipart(text/plain))', 1),\n",
       " ('multipart(text/plain, application/x-pkcs7-signature)', 1),\n",
       " ('multipart(text/plain, multipart(text/plain, text/plain), text/rfc822-headers)',\n",
       "  1),\n",
       " ('multipart(text/plain, multipart(text/plain, text/plain), multipart(multipart(text/plain, application/x-pkcs7-signature)))',\n",
       "  1),\n",
       " ('multipart(text/plain, application/x-java-applet)', 1)]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "structures_counter(ham_emails).most_common() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('text/plain', 219),\n",
       " ('text/html', 183),\n",
       " ('multipart(text/plain, text/html)', 45),\n",
       " ('multipart(text/html)', 20),\n",
       " ('multipart(text/plain)', 19),\n",
       " ('multipart(multipart(text/html))', 5),\n",
       " ('multipart(text/plain, image/jpeg)', 3),\n",
       " ('multipart(text/html, application/octet-stream)', 2),\n",
       " ('multipart(text/plain, application/octet-stream)', 1),\n",
       " ('multipart(text/html, text/plain)', 1),\n",
       " ('multipart(multipart(text/html), application/octet-stream, image/jpeg)', 1),\n",
       " ('multipart(multipart(text/plain, text/html), image/gif)', 1),\n",
       " ('multipart/alternative', 1)]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "structures_counter(spam_emails).most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Return-Path : <12a1mailbot1@web.de>\n",
      "Delivered-To : zzzz@localhost.spamassassin.taint.org\n",
      "Received : from localhost (localhost [127.0.0.1])\tby phobos.labs.spamassassin.taint.org (Postfix) with ESMTP id 136B943C32\tfor <zzzz@localhost>; Thu, 22 Aug 2002 08:17:21 -0400 (EDT)\n",
      "Received : from mail.webnote.net [193.120.211.219]\tby localhost with POP3 (fetchmail-5.9.0)\tfor zzzz@localhost (single-drop); Thu, 22 Aug 2002 13:17:21 +0100 (IST)\n",
      "Received : from dd_it7 ([210.97.77.167])\tby webnote.net (8.9.3/8.9.3) with ESMTP id NAA04623\tfor <zzzz@spamassassin.taint.org>; Thu, 22 Aug 2002 13:09:41 +0100\n",
      "From : 12a1mailbot1@web.de\n",
      "Received : from r-smtp.korea.com - 203.122.2.197 by dd_it7  with Microsoft SMTPSVC(5.5.1775.675.6);\t Sat, 24 Aug 2002 09:42:10 +0900\n",
      "To : dcek1a1@netsgo.com\n",
      "Subject : Life Insurance - Why Pay More?\n",
      "Date : Wed, 21 Aug 2002 20:31:57 -1600\n",
      "MIME-Version : 1.0\n",
      "Message-ID : <0103c1042001882DD_IT7@dd_it7>\n",
      "Content-Type : text/html; charset=\"iso-8859-1\"\n",
      "Content-Transfer-Encoding : quoted-printable\n"
     ]
    }
   ],
   "source": [
    "for header,value in spam_emails[0].items():\n",
    "    print(header,\":\",value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Life Insurance - Why Pay More?'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spam_emails[0][\"Subject\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 拆分训练集和测试集合"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X = np.array(ham_emails+spam_emails)\n",
    "y = np.array([0]*len(ham_emails)+[1]*len(spam_emails))\n",
    "\n",
    "X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.2,random_state = 42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "首先需要一个函数来将html转化为纯文本，使用【beautifulsoup】库，下面的函数首先删除<head>部分，然后将所有<a>标记转换为单词hyperlink，然后去掉搜有html标记，只留下纯文本。\n",
    "为了可读性，他还用一个换行符替换多个黄行服最后他取消了HTML实体\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from html import unescape\n",
    "def html_to_plain_text(html):\n",
    "    text = re.sub('<head.*?>.*?</head>', '', html, flags=re.M | re.S | re.I)\n",
    "    text = re.sub('<a\\s.*?>', ' HYPERLINK ', text, flags=re.M | re.S | re.I)\n",
    "    text = re.sub('<.*?>', '', text, flags=re.M | re.S)\n",
    "    text = re.sub(r'(\\s*\\n)+', '\\n', text, flags=re.M | re.S)\n",
    "    return unescape(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<HTML><HEAD><TITLE>MILFhunter</TITLE>\n",
      "<META http-equiv=Content-Type\n",
      "content=\"text/html; charset=windows-1252\"><HTTP-EQUIV=\"PRAGMA\"\n",
      "CONTENT=\"NO-CACHE\">\n",
      "<SCRIPT language=JavaScript>\n",
      "<!-- hide from old browsers\n",
      "\tfunction loadPage(pageURL){\n",
      "\tlocation.href = pageURL.options[pageURL.selectedIndex].value\n",
      "\t}\n",
      "//-->\n",
      "</SCRIPT>\n",
      "\n",
      "<META content=\"MSHTML 6.00.2716.2200\" name=GENERATOR></HEAD>\n",
      "<BODY text=#eaebec vLink=#ffffcc aLink=#ffffff link=#ffffcc bgColor=#647481\n",
      "leftMargin=0 background=\"http://www.fromyou2.com/nasty/milf/bg.jpg\"\n",
      "topMargin=0>\n",
      "<CENTER><BR>\n",
      "  <CENTER>\n",
      "    <CENTER>\n",
      "      <FONT face=verdana><BR>\n",
      "      </FONT>\n",
      "      <CENTER>\n",
      "        <TABLE cellPadding=15 bgColor=gray>\n",
      "          <TBODY>\n",
      "          <TR>\n",
      "            <TD>\n",
      "              <CENTER>\n",
      "                <font color=\"black\" face=\"verdana\"><A\n",
      "      onmouseover=\"window.status='MILFhunter.com - Do you know where your mom is?';return true\"\n",
      "      href=\"http://www.fromyou2.com/nasty/milf/milf/bindex.htm\"><IMG\n",
      "      src=\"http://www.fromyou2. ...\n"
     ]
    }
   ],
   "source": [
    "html_spam_emails = [email for email in X_train[y_train ==1]\n",
    "                   if get_email_structure(email) == 'text/html']\n",
    "sample_html_spam = html_spam_emails[7]\n",
    "print(sample_html_spam.get_content().strip()[:1000],\"...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "                 HYPERLINK\n",
      "                     HYPERLINK\n",
      "                     HYPERLINK\n",
      "                     HYPERLINK\n",
      "                     HYPERLINK\n",
      "                     HYPERLINK\n",
      "                     HYPERLINK\n",
      "                     HYPERLINK\n",
      "                     HYPERLINK\n",
      "                     HYPERLINK\n",
      "                     HYPERLINK\n",
      "                     HYPERLINK\n",
      "                     HYPERLINK\n",
      "                     HYPERLINK\n",
      "                     HYPERLINK\n",
      "                     HYPERLINK\n",
      "                     HYPERLINK\n",
      "                 HYPERLINK MILF HUNTER\n",
      "                Do you know where your mom is?\n",
      "                 HYPERLINK\n",
      "                MORE SAMPLE PICS      MORE SAMPLE MOVIES      LIST OF MILFs\n",
      "         \n",
      "         HYPERLINK CLICK\n",
      "          HERE to enlarge your PENIS 3-4 inches NATURALLY!!\n",
      "         \n",
      "         \n",
      "         HYPERLINK Click\n",
      "          Here to be removed\n",
      " ...\n"
     ]
    }
   ],
   "source": [
    "print(html_to_plain_text(sample_html_spam.get_content())[:1000],\"...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def email_to_text(email):\n",
    "    html = None\n",
    "    for part in email.walk():\n",
    "        ctype = part.get_content_type()\n",
    "        if not ctype in (\"text/plain\",\"text/html\"):\n",
    "            continue\n",
    "        try:\n",
    "            content = part.get_content()\n",
    "        except:\n",
    "            content = str(part.get_payload())\n",
    "        if ctype == \"text/plain\":\n",
    "            return content\n",
    "        else:\n",
    "            html = content\n",
    "    if html:\n",
    "        return html_to_plain_text(html)\n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "                 HYPERLINK\n",
      "                     HYPERLINK\n",
      "                     HYPERLINK\n",
      "           ...\n"
     ]
    }
   ],
   "source": [
    "print(email_to_text(sample_html_spam)[:100],\"...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/xuzy/opt/anaconda3/lib/python3.7/site-packages/sklearn/feature_extraction/text.py:17: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working\n",
      "  from collections import Mapping, defaultdict\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "from urlextract import URLExtract"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator,TransformerMixin\n",
    "\n",
    "class EmailToWordCounterTransformer(BaseEstimator,TransformerMixin):\n",
    "    def __init__(self,strip_headers = True,lower_case = True,remove_punctuation=True,\n",
    "                replace_urls=True,replace_numbers = True,stemming=True):\n",
    "        self.strip_header = strip_headers\n",
    "        self.lower_case = lower_case\n",
    "        self.remove_punctuation = remove_punctuation\n",
    "        self.replace_urls = replace_urls\n",
    "        self.replace_numbers = replace_numbers\n",
    "        self.stemming = stemming\n",
    "    def fit(self,X,y=None):\n",
    "        return self\n",
    "    def transform(self,X,y = None):\n",
    "        X_transformed = []\n",
    "        for email in X:\n",
    "            text = email_to_text(email) or \"\"\n",
    "            if self.lower_case:\n",
    "                text = text.lower()\n",
    "            if self.replace_urls:\n",
    "                extractor = URLExtract()\n",
    "                urls = list(set(extractor.find_urls(text)))\n",
    "                urls.sort(key=lambda url:len(url),reverse =True)\n",
    "                for url in urls:\n",
    "                    text = text.replace(url,\"URL\")\n",
    "            if self.replace_numbers:\n",
    "                text = re.sub(r'\\d+(?:\\.\\d*(?:[eE]\\d))','NUMBER',text)\n",
    "            if self.remove_punctuation:\n",
    "                text = re.sub(r'\\W+',' ',text,flags = re.M)\n",
    "            word_counts = Counter(text.split())\n",
    "#                 if self.stemming:\n",
    "#                     stemmed_word_counts = Counter()\n",
    "#                     for word,count in word_counts.items():\n",
    "#                         stemmed_word = stemmer.stem(word)\n",
    "#                         stemmed_word_counts[stemmed_word] +=count\n",
    "#                     word_counts = stemmed_word_counts\n",
    "            X_transformed.append(word_counts)\n",
    "        return np.array(X_transformed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([Counter({'the': 11, 'of': 9, 'and': 8, 'all': 3, 'christianity': 3, 'to': 3, 'by': 3, 'jefferson': 2, 'i': 2, 'have': 2, 'one': 2, 'on': 2, 'been': 2, 'has': 2, 'half': 2, 'jesus': 2, 'some': 1, 'interesting': 1, 'quotes': 1, 'URL': 1, 'thomas': 1, 'examined': 1, 'known': 1, 'superstitions': 1, 'word': 1, 'do': 1, 'not': 1, 'find': 1, 'in': 1, 'our': 1, 'particular': 1, 'superstition': 1, 'redeeming': 1, 'feature': 1, 'they': 1, 'are': 1, 'alike': 1, 'founded': 1, 'fables': 1, 'mythology': 1, 'millions': 1, 'innocent': 1, 'men': 1, 'women': 1, 'children': 1, 'since': 1, 'introduction': 1, 'burnt': 1, 'tortured': 1, 'fined': 1, 'imprisoned': 1, 'what': 1, 'effect': 1, 'this': 1, 'coercion': 1, 'make': 1, 'world': 1, 'fools': 1, 'other': 1, 'hypocrites': 1, 'support': 1, 'roguery': 1, 'error': 1, 'over': 1, 'earth': 1, 'six': 1, 'historic': 1, 'americans': 1, 'john': 1, 'e': 1, 'remsburg': 1, 'letter': 1, 'william': 1, 'short': 1, 'again': 1, 'become': 1, 'most': 1, 'perverted': 1, 'system': 1, 'that': 1, 'ever': 1, 'shone': 1, 'man': 1, 'rogueries': 1, 'absurdities': 1, 'untruths': 1, 'were': 1, 'perpetrated': 1, 'upon': 1, 'teachings': 1, 'a': 1, 'large': 1, 'band': 1, 'dupes': 1, 'importers': 1, 'led': 1, 'paul': 1, 'first': 1, 'great': 1, 'corrupter': 1, 'teaching': 1}),\n",
       "       Counter({'URL': 4, 's': 3, 'to': 3, 'in': 2, 'forteana': 2, 'martin': 2, 'an': 2, 'and': 2, 'we': 2, 'is': 2, 'yahoo': 2, 'groups': 2, 'unsubscribe': 2, 'y': 1, 'adamson': 1, 'wrote': 1, 'for': 1, 'alternative': 1, 'rather': 1, 'more': 1, 'factually': 1, 'based': 1, 'rundown': 1, 'on': 1, 'hamza': 1, 'career': 1, 'including': 1, 'his': 1, 'belief': 1, 'that': 1, 'all': 1, 'non': 1, 'muslims': 1, 'yemen': 1, 'should': 1, 'be': 1, 'murdered': 1, 'outright': 1, 'know': 1, 'how': 1, 'unbiased': 1, 'memri': 1, 'don': 1, 't': 1, 'html': 1, 'rob': 1, 'sponsor': 1, '4': 1, 'dvds': 1, 'free': 1, 'p': 1, 'join': 1, 'now': 1, 'from': 1, 'this': 1, 'group': 1, 'send': 1, 'email': 1, 'egroups': 1, 'com': 1, 'your': 1, 'use': 1, 'of': 1, 'subject': 1}),\n",
       "       Counter({'i': 6, 'the': 6, 'kernel': 6, 'source': 4, '2': 4, '4': 4, '18': 4, '3': 4, 'of': 3, 'get': 3, 'is': 3, 'tell': 3, 'URL': 3, 'list': 3, 'build': 2, 'scripts': 2, 'some': 2, 'package': 2, 'on': 2, 'apt': 2, 'install': 2, 'this': 2, 'can': 2, 'to': 2, 'thomas': 2, 'rpm': 2, 'hi': 1, 'in': 1, 'my': 1, 'have': 1, 'problems': 1, 'with': 1, 'packages': 1, 'for': 1, 'sources': 1, 'a': 1, 'virtual': 1, 'provided': 1, 'by': 1, 'running': 1, 'now': 1, 'first': 1, 'all': 1, 'doesn': 1, 't': 1, 'really': 1, 'me': 1, 'what': 1, 'two': 1, 'options': 1, 'are': 1, 'second': 1, 'there': 1, 'way': 1, 'either': 1, 'done': 1, 'from': 1, 'automatic': 1, 'so': 1, 'd': 1, 'like': 1, 'it': 1, 'proceed': 1, 'anyway': 1, 'thanks': 1, 'dave': 1, 'dina': 1, 'project': 1, 'future': 1, 'tv': 1, 'today': 1, 'm': 1, 'alive': 1, 'because': 1, 'pain': 1, 'apestaart': 1, 'org': 1, 'urgent': 1, 'best': 1, 'radio': 1, 'internet': 1, '24': 1, '7': 1, '_______________________________________________': 1, 'mailing': 1, 'freshrpms': 1, 'net': 1})],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_few = X_train[:3]\n",
    "X_few_wordcounts = EmailToWordCounterTransformer().fit_transform(X_few)\n",
    "X_few_wordcounts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "有了单词计数，我们需要把它们转换成向量。为此，我们将构建另一个转换器，其fit()方法将构建词汇表（最常用单词的有序列表），其transform（）方法将使用词汇表将单词计数转换为向量。输出是稀疏矩阵"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.sparse import csr_matrix\n",
    "\n",
    "class WordCounterToVectorTransformer(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, vocabulary_size=1000):\n",
    "        self.vocabulary_size = vocabulary_size  # 词汇量\n",
    "    def fit(self, X, y=None):\n",
    "        total_count = Counter()\n",
    "        for word_count in X:\n",
    "            for word, count in word_count.items():\n",
    "                total_count[word] += min(count, 10)\n",
    "        most_common = total_count.most_common()[:self.vocabulary_size]\n",
    "        self.most_common_ = most_common\n",
    "        self.vocabulary_ = {word: index + 1 for index, (word, count) in enumerate(most_common)}\n",
    "        return self\n",
    "    def transform(self, X, y=None):\n",
    "        rows = []\n",
    "        cols = []\n",
    "        data = []\n",
    "        for row, word_count in enumerate(X):\n",
    "            for word, count in word_count.items():\n",
    "                rows.append(row)\n",
    "                cols.append(self.vocabulary_.get(word, 0))\n",
    "                data.append(count)\n",
    "        return csr_matrix((data, (rows, cols)), shape=(len(X), self.vocabulary_size + 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from scipy.sparse import *\n",
    "# row = [0,0,0,1,1,1,2,2,2]\n",
    "# col = [0,1,2,0,1,2,0,1,2]\n",
    "# data = [1,0,1,0,1,1,1,1,0]\n",
    "# team = csr_matrix((data,(row,col)),shape(3,3))\n",
    "# print(team)\n",
    "# print(team.todense())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<3x11 sparse matrix of type '<class 'numpy.longlong'>'\n",
       "\twith 27 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_transformer = WordCounterToVectorTransformer(vocabulary_size=10)\n",
    "X_few_vectors = vocab_transformer.fit_transform(X_few_wordcounts)\n",
    "X_few_vectors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 我们现在准备训练第一个垃圾邮件分类器\n",
    "## 转换数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "preprocess_pipeline = Pipeline([\n",
    "    (\"email_tp_wordcount\",EmailToWordCounterTransformer()),\n",
    "    (\"wordcount_to_vector\",WordCounterToVectorTransformer()),\n",
    "])\n",
    "\n",
    "X_train_transformed = preprocess_pipeline.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  ................................................................\n",
      "[CV] ........................ , score=0.978776529338327, total=   0.1s\n",
      "[CV]  ................................................................\n",
      "[CV] ..................................... , score=0.98, total=   0.1s\n",
      "[CV]  ................................................................\n",
      "[CV] ..................................... , score=0.99, total=   0.1s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.3s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9829255097794424"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "log_clf = LogisticRegression(solver=\"liblinear\",random_state=42)\n",
    "score = cross_val_score(log_clf,X_train_transformed,y_train,cv=3,verbose=3)\n",
    "score.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "精度:98.94%\n",
      "召回:89.42%\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score,recall_score\n",
    "\n",
    "X_test_transformed = preprocess_pipeline.transform(X_test)\n",
    "\n",
    "log_clf = LogisticRegression(solver = \"liblinear\",random_state=42)\n",
    "log_clf.fit(X_train_transformed,y_train)\n",
    "\n",
    "y_pred = log_clf.predict(X_test_transformed)\n",
    "\n",
    "print(\"精度:{:.2f}%\".format(100*precision_score(y_test,y_pred)))\n",
    "print(\"召回:{:.2f}%\".format(100 * recall_score(y_test,y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
